// SPDX-License-Identifier: GPL-2.0-only

#include <asm/kvm_pgtable.h>
#include <asm/neon-intrinsics.h>

#define VECTOR_SIZE			8

void vmark_page_dirty(struct kvm *kvm, kvm_pte_t *ptep, u64 addr, u64 end)
{
	uint16x8x4_t vdata;
	u32 n, idx = (addr >> PAGE_SHIFT) & (BIT(PAGE_SHIFT - 3) - 1);

	for (; idx < PTRS_PER_PTE; idx += VECTOR_SIZE) {
		if (addr >= end)
			return;

		vdata = vld4q_u16((u16 *)(ptep + idx));
		for (n = 0; n < VECTOR_SIZE; n++, addr += PAGE_SIZE) {
			if (addr >= end || (n + idx >= PTRS_PER_PTE))
				return;

			if (!kvm_pte_valid(vdata.val[0][n]))
				continue;

			if (stage2_pte_writable(vdata.val[0][n])) {
				mark_page_dirty(kvm, addr >> PAGE_SHIFT);
				mark_hwdbm_dirty(kvm, addr >> PAGE_SHIFT, false);
			}
		}
	}
}
